from __future__ import division
from operator import itemgetter, attrgetter
from struct import *
import gc
import math
import os
import random
import sys
import time


# Let's make it suitable for vidaserver1
def compute_verify_overlap_for_Xdoc_method(NUM_DOCS,ofh,queryProcessingMethod,ofn):
    # vidaserver1
    inputFileName1 = "/home/vgc/wei/workspace/NYU_IRTK/data/gov2_Docs_with_TheirXdocValues_Since20140428_sortedByXdocValues"
    # dodo (crash)
    # inputFileName1 = "/home/diaosi/gov2ClearYourMindAndDoItAgain/gov2_Docs_with_TheirXdocValues_Since20140428_sortedByXdocValues"
    inputFileHanlder = open(inputFileName1,"r")
    currentLine = inputFileHanlder.readline()
    
    docIDDict = {}
    while currentLine:
        currentLineElements = currentLine.strip().split(" ")
        currentDocID = currentLineElements[1]
        if currentDocID not in docIDDict:
            docIDDict[currentDocID] = 1
        else:
            print "duplicated docID detected."
            exit(1)
        if len(docIDDict) == NUM_DOCS:
            # 16234 1%
            # 58892 3%
            # 113151 5%
            # 303078 10%
            # 573053 15%
            # 960415 20%
            # 1704697 30%
            # 3048106 40%
            # 4840867 50%
            # 7078881 60%
            # 9726244 70%
            # 13270243 80%
            # 17993779 90%
            # 25205179 100%
            break
        currentLine = inputFileHanlder.readline()
    print "len(docIDDict):",len(docIDDict)
    inputFileHanlder.close()
      
    # on vidaserver1:
    inputFileName2 = ""
    # AND case
    if queryProcessingMethod == "AND":
    	inputFileName2 = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/rawResults_TOP1000_AND_testingQueries_20140827"
    # OR case
    elif queryProcessingMethod == "OR":
    	inputFileName2 = "/home/vgc/wei/workspace/NYU_IRTK/data/rawResults_50%_TOP10_OR_20140126Night"
    else:
    	print "NO query processing method selected."
    	exit(1)
    inputFileHanlder2 = open(inputFileName2,"r")
    currentQueryID = ""
    currentLine = inputFileHanlder2.readline()
    numOfTOP10DocumentResults = 0
    numOfTOP10DocumentResultsCaptured = 0
    numOfTOP10DocumentResultsCapturedList = []
    currentRank = 0
    NUM_QUERIES_NEEDED_TO_PROCESSED = 101
    num_queries_processed = 0
    currentQueryNumOfQueriesInTotal = 0
    currentQueryNumOfQueriesRetrieved = 0
    while currentLine:
        currentLineElements = currentLine.strip().split(" ")
        if currentLine.strip().startswith("qid:"):
        	# The previous query
        	# print currentQueryID
            # print currentLineElements[1]
            if currentQueryID.strip() != "":
	            print currentQueryID,currentQueryNumOfQueriesRetrieved,currentQueryNumOfQueriesInTotal
	            ofh.write(str(currentQueryID) + " " + str(currentQueryNumOfQueriesRetrieved) + " " + str(currentQueryNumOfQueriesInTotal) + "\n")
            currentQueryNumOfQueriesRetrieved = 0
            currentQueryNumOfQueriesInTotal = 0
            currentQueryID = currentLineElements[1]
            currentRank = 0
            num_queries_processed += 1
            #if num_queries_processed == NUM_QUERIES_NEEDED_TO_PROCESSED:
        	#	break
        	
        if len(currentLineElements) == 25 and currentLineElements[-2].startswith("GX"):
            currentRank = int(currentLineElements[0])
            if currentRank <= 10:
            	currentQueryNumOfQueriesInTotal += 1
                # print currentRank,currentLine.strip()
                currentDocID = currentLineElements[-3]
                if currentDocID in docIDDict:
                    numOfTOP10DocumentResultsCaptured += 1
                    currentQueryNumOfQueriesRetrieved += 1
                    # print "----->",currentQueryID,currentDocID
                    numOfTOP10DocumentResultsCapturedList.append(currentDocID)
                numOfTOP10DocumentResults += 1
        currentLine = inputFileHanlder2.readline()
    
    # final qid
    print currentQueryID,currentQueryNumOfQueriesRetrieved,currentQueryNumOfQueriesInTotal
    ofh.write(str(currentQueryID) + " " + str(currentQueryNumOfQueriesRetrieved) + " " + str(currentQueryNumOfQueriesInTotal) + "\n")
    print "numOfTOP10DocumentResults:",numOfTOP10DocumentResults
    print "numOfTOP10DocumentResultsCaptured:",numOfTOP10DocumentResultsCaptured
    print "inputFileName1:",inputFileName1
    print "inputFileName2:",inputFileName2
    print "ofn:",ofn
    # print "numOfTOP10DocumentResultsCapturedList:",numOfTOP10DocumentResultsCapturedList

print "Begins..."
'''
print 'Number of arguments:', len(sys.argv), 'arguments.'
print 'Argument List:', str(sys.argv)
numOfDocumentsIncluded = 0
if len(sys.argv) != 2:
    print "USE python programName.py # of documents included"
    exit(1)
else:
    numOfDocumentsIncluded = int(sys.argv[1])
    print "# of documents included:",numOfDocumentsIncluded
'''

# differencesOfResultsBetweenANDAndORSemanticsOf95KQueries()
# differencesOfResultsBetweenANDAndORSemanticsOfHJQueries()
# getResultsAndComputePAt10AtDifferentPruningLevels(numOfDocumentsIncluded) # This function is talking about quality
# getResultsAndComputeOverlapAgainstUnprunedIndex()
# Updated by Wei 20140904
# 16234 1%
# 58892 3%
# 113151 5%
# 303078 10%
# 573053 15%
# 960415 20%
# 1704697 30%
# 3048106 40%
# 4840867 50%
# 7078881 60%
# 9726244 70%
# 13270243 80%
# 17993779 90%
# 25205179 100%
NUM_DOCS = int(sys.argv[1])
PERCENTAGE_STR = sys.argv[2]
queryProcessingMethod = sys.argv[3]
ofn = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/tiering/xDoc_documentAllocationRevisited_since20140904/raw_xDoc_firstTierGain_" + str(PERCENTAGE_STR) + "_" + queryProcessingMethod
ofh = open(ofn,"w")
compute_verify_overlap_for_Xdoc_method(NUM_DOCS,ofh,queryProcessingMethod,ofn)
ofh.close()
print "Ends."
